# -*- codeing = utf-8 -*-
# @Time : 2024/5/31 20:38
# @Author : huangjing
# @File : spider.py
# @Software : PyCharm
from bs4 import BeautifulSoup
import re
import urllib.request
import urllib.parse
import urllib.response
import xlwt

# 获取指定的url网页内容
def get_html(url):
    # 模拟浏览器头部的信息
    head = {
        "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/91.0.4472.114 Safari/537.36"
    }
    request = urllib.request.Request(url, headers=head)
    html = ""
    try:
        response = urllib.request.urlopen(request)
        html = response.read().decode('utf-8')
    except urllib.error.URLError as e:
        if hasattr(e, "code"):
            print(e.code)
        if hasattr(e, "reason"):
            print(e.reason)
    return html
# 正则表达式获取 API名称
re_api_name = re.compile(r'torch\.[A-Za-z0-9_\.\*\(\)\-\>\,\=\s]*\\u003C')
# 字符串切分，去掉最后一个元素，返回列表
def get_list(s:str, tag:str) -> list:
    tag_list = s.split(tag)
    tag_list.pop()
    return tag_list

# 爬取数据
def get_data_pytorch(baseurl):
    html = get_html(baseurl)
    # 1.逐一解析数据
    soup = BeautifulSoup(html, "html.parser")
    body = soup.find_all('body')[0]
    context = str(body)
    context = context.replace("&gt;", ">")
    context = context.replace("&nbsp;", " ")
    # re库用来通过正则表达式查找指定的字符串
    api_name_list = re.findall(re_api_name, context)
    api_name_list = list(map(lambda x: x.split('\\u')[0], api_name_list))
    print(api_name_list)
    print(len(api_name_list))
    return api_name_list

def get_data_mindtorch(baseurl):
    html = get_html(baseurl)
    api_name_list = []
    # 1.逐一解析数据
    soup = BeautifulSoup(html, "html.parser")
    div = soup.find_all('div', attrs={'class': 'markdown-content'})[0]
    table_list = div.find_all('table')
    for table in table_list:
        th = table.thead.tr.th.text
        api_name_list.append(th)
        tr_list = table.tbody.find_all('tr')
        for tr in tr_list:
            api_name_list.append(tr.td.text)
    print(api_name_list)
    print(len(api_name_list))
    return api_name_list
# 保存数据
def saveData(datalist, savepath):
    book = xlwt.Workbook(encoding='utf-8', style_compression=0)
    sheet = book.add_sheet("torch", cell_overwrite_ok=True)
    # col = ("API名称", "是否支持", "限制与说明")
    # 将列名写入excel
    sheet.write(0, 0, "API名称")
    for i in range(0, len(datalist)):
        print("第%d条" % (i + 1))
        sheet.write(i + 1, 0, datalist[i])
    book.save(savepath)
    print("加载完毕！")

if __name__ == '__main__':
    # pythort
    url_pytorch = r"https://www.hiascend.com/document/detail/zh/canncommercial/70RC1/modeldevpt/ptmigr/ptaoplist_064.html"
    data_list = get_data_pytorch(url_pytorch)
    save_path = "torch.xls"
    saveData(data_list, save_path)
    # mindtorch
    url_mindtorch = r"https://openi.pcl.ac.cn/OpenI/MSAdapter/src/branch/master/doc/readthedocs/source_zh/docs/SupportedList.md#user-content-span-id-jump16-torch-distributed-span"
    data_list = get_data_mindtorch(url_mindtorch)
    save_path = "mindtorch.xls"
    saveData(data_list, save_path)
